# -*- coding: utf-8 -*-
# @Time    : 2020/9/3 15:14
# @Author  : suhong
# @File    : paper_mg_2_db3.py
# @Software: PyCharm
import re
import sqlite3
import time

from re_common.baselibrary.utils.basedir import BaseDir

from re_common.baselibrary.utils.basefile import BaseFile
from re_common.baselibrary.utils.basepymongo import BasePyMongo


class Paper2Db3(object):
    def __init__(self):
        self.basemongo = BasePyMongo(
            "mongodb://192.168.31.30:32417/htmljson.paper_jinjiribao?authSource=htmljson")
        self.basemongo.use_db("htmljson")
        self.basemongo.create_col("a_paper_latest_test")
        self.now_date = time.strftime('%Y%m%d', time.localtime())

    def clean_author(self, author):
        new_author = ""
        author = author.replace("海外网", "").replace("特约记者", "").replace("文/图", "").replace("/绘", "").replace("/文",
                                                                                                            "").replace(
            "/图", "").replace("本报驻联合国记者", "").replace("本报记者", "").replace("新华社记者", "").replace("□", "").replace("■",
                                                                                                                "").replace(
            "本报评论员", "").replace("采访组：", "").replace("\n", "").replace("字数:", "").replace("通讯员", "").replace("农村金融时报记者",
                                                                                                             "").replace(
            "经济日报·中国经济网记者", "").replace("　", ";")
        pat = '本.*?记者'
        xx = re.findall(pat, author)
        for item in xx:
            author = author.replace(item, "")

        author = author.replace(" ", ";").strip(";")
        if ";;" in author:
            author = author.replace(";;", ";")
            if author.startswith(";"):
                author = author[1:]
        string = ""
        for i in author.split(";"):
            if len(i) > 1:
                new_author += i + ";"
                continue
            else:
                string += i
                if len(string) >= 2:
                    new_author += string + ";"
                    string = ""
        new_author = new_author.strip(";")
        return new_author

    def get_db3(self, journal_name):
        file_path = r'E:\download\paper\{}'.format(self.now_date)
        BaseDir.create_dir(file_path)
        db3_path = file_path + '/' + journal_name + '.db3'
        if not BaseFile.is_file_exists(db3_path):
            BaseFile.copy_file_to_file(r"E:\db3\papers/temp.db3", db3_path)

        conn = sqlite3.connect(db3_path, check_same_thread=False)
        cur = conn.cursor()
        sql = "replace INTO paper_info(lngid,years,title_c,showwriter,publishdate,bznewspaper,bzcolumn,bzkicker,bzsubtitle,bzpane,bzsort,language,remark) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"
        slist = list()

        for item in self.basemongo.find({"journal_name": journal_name}):
            lngid = item['lngid_alt']
            years = item['pub_year']
            title_c = item['title']
            showwriter = self.clean_author(item['authors'])
            publishdate = item['pub_date']
            bznewspaper = item['journal_name']

            bzkicker = item['title_catalyst']

            bzsubtitle = item['title_alt']
            bzpane = item['versions_title']
            bzsort = item['index']
            language = item['language']
            remark = item['abstract']
            down_date = item['down_date']

            # 处理栏目
            bzcolumn = ""
            if remark.startswith("【"):
                patt = r'【(.*?)】'
                xx = re.findall(patt, remark)
                if xx:
                    bzcolumn = xx[0]

            # if down_date == '20201116':
            #     slist.append((lngid, years, title_c, showwriter, publishdate, bznewspaper, bzcolumn, bzkicker,
            #                   bzsubtitle, bzpane, bzsort, language, remark))

            if publishdate > '20201020':
                slist.append((lngid,years,title_c,showwriter,publishdate,bznewspaper,bzcolumn,bzkicker,bzsubtitle,bzpane,bzsort,language,remark))


            if len(slist) > 30000:
                cur.executemany(sql, slist)
                conn.commit()
                print("插入{}条成功".format(len(slist)))
                slist.clear()

        cur.executemany(sql, slist)
        conn.commit()
        print("插入{}条成功".format(len(slist)))
        slist.clear()


if __name__ == '__main__':
    for journal_name in ['北京日报']:
        # 北京日报
        # for journal_name in ['解放军报','经济日报','人民日报','人民日报海外版','学习时报','光明日报']:
        print(journal_name)
        p = Paper2Db3()
        p.get_db3(journal_name)
